#! /bin/sh

# This script is intended to simulate loss and recovery of two
# EC2 nodes out of a pool of three, with an artarded operator
# that doesn't delete the schema directory of a stale node.

ERL_CRASH_DUMP=${ERL_CRASH_DUMP-/dev/null}
export ERL_CRASH_DUMP

pid=$$
cook=mega$$

cleanup () \
  {
    erl -pa ../src -setcookie $cook -sname killah$pid -noshell -noinput -eval '
      Pid = hd (init:get_plain_arguments ()),
      Host = lists:last (string:tokens (atom_to_list (node ()), "@")),
      rpc:call (list_to_atom ("flassy" ++ Pid ++ "@" ++ Host), erlang, halt, []),
      rpc:call (list_to_atom ("turgy" ++ Pid ++ "@" ++ Host), erlang, halt, []),
      rpc:call (list_to_atom ("warezy" ++ Pid ++ "@" ++ Host), erlang, halt, []),
      rpc:call (list_to_atom ("inviso" ++ Pid ++ "@" ++ Host), erlang, halt, [])
    ' -s erlang halt -extra $pid

    rm -rf Mnesia*flassy*
    rm -rf Mnesia*turgy*
    rm -rf Mnesia*warezy*
    rm -rf Mnesia*inviso*
  }

trap 'cleanup;' EXIT
trap 'exit 1;' INT QUIT TERM HUP

start_bg_node () \
  {
    erl -pa ../src -setcookie $cook -sname "$1" -schemafinder foreign_key_bugfix false -s combonodefinder -s schemafinder $2 -noshell -noinput -eval '
      true = register (hello, self ()),
      { atomic, ok } = delirium:heartbeat (),
      true = lists:keymember (schemafinder, 1, application:which_applications ()),
      receive { From, ruthere } -> From ! imok end,
      receive after infinity -> ok end
    ' -s erlang halt &

    wait_for_node "$1"
  }

wait_for_node () \
  {
    erl -hidden -pa ../src -setcookie $cook -sname wazzup$pid      \
        -noshell -noinput -eval '
      receive after 1000 -> ok end,
      Host = lists:last (string:tokens (atom_to_list (node ()), "@")),
      Other = hd (init:get_plain_arguments ()),
      While = fun (F, W) -> case F () of false -> ok; 
                                         true -> receive after 1000 -> ok end,
                                                 W (F, W)
                            end
              end,
      While (fun () -> pang =:= net_adm:ping (list_to_atom (Other ++ "@" ++ Host)) end, While),
      While (fun () -> rpc:call (list_to_atom (Other ++ "@" ++ Host), erlang, whereis, [ hello ]) =:= undefined end, While),
      Pid = rpc:call (list_to_atom (Other ++ "@" ++ Host), erlang, whereis, [ hello ]),
      true = erlang:is_pid (Pid),
      MRef = erlang:monitor (process, Pid),
      Pid ! { self (), ruthere },
      ok = receive imok -> ok; { 'DOWN', MRef, _, _, _ } -> flass end
    ' -s erlang halt -extra "$1" || exit 1
  }

{
rm -rf Mnesia*flassy*
rm -rf Mnesia*turgy*
rm -rf Mnesia*warezy*
rm -rf Mnesia*inviso*

# first we start three nodes and get them running the same mnesia schema
# plus a fourth in a different group that nobodoy is supposed to see ...

start_bg_node inviso$pid '-schemafinder group flass'
start_bg_node flassy$pid
start_bg_node turgy$pid

erl -pa ../src -setcookie $cook -sname warezy$pid -s combonodefinder -schemafinder foreign_key_bugfix false -s schemafinder -noshell -noinput -eval '
  true = register (hello, self ()),
  { atomic, ok } = delirium:heartbeat (),
  true = lists:keymember (schemafinder, 1, application:which_applications ()),
  receive after 1000 -> ok end,
  length (mnesia:system_info (running_db_nodes)) =:= 2,
  { atomic, ok } = 
    mnesia:create_table (flass,
                         [ { disc_copies,
                             mnesia:system_info (running_db_nodes)  } ]),
  receive { From, ruthere } -> From ! imok end,
  receive after infinity -> ok end
' -s erlang halt &

wait_for_node warezy$pid

# now kill flassy and turgy 

erl -hidden -pa ../src -setcookie $cook -sname killah$pid -noshell -noinput -eval '
  Pid = hd (init:get_plain_arguments ()),
  Host = lists:last (string:tokens (atom_to_list (node ()), "@")),
  rpc:call (list_to_atom ("flassy" ++ Pid ++ "@" ++ Host), erlang, halt, []),
  rpc:call (list_to_atom ("turgy" ++ Pid ++ "@" ++ Host), erlang, halt, []),
  { atomic, ok } = rpc:call (list_to_atom ("warezy" ++ Pid ++ "@" ++ Host),
                             delirium,
                             condemn,
                             [ list_to_atom ("flassy" ++ Pid ++ "@" ++ Host) ]),
  { atomic, ok } = rpc:call (list_to_atom ("warezy" ++ Pid ++ "@" ++ Host),
                             delirium,
                             condemn,
                             [ list_to_atom ("turgy" ++ Pid ++ "@" ++ Host) ]),
  { ok, Prune } = rpc:call (list_to_atom ("warezy" ++ Pid ++ "@" ++ Host), 
                            delirium,
                            force_prune,
                            []),
  MonitorRef = erlang:monitor (process, Prune),
  receive
    { 'DOWN', MonitorRef, _Type, _Object, _Info } -> ok
  end,
  Warez = list_to_atom ("warezy" ++ Pid ++ "@" ++ Host),
  [ Warez ] = rpc:call (Warez, mnesia, system_info, [ db_nodes ])
' -s erlang halt -extra $pid || exit 1

# now restart turgy

rm -rf Mnesia*turgy*

erl -pa ../src -setcookie $cook -sname turgy$pid -s combonodefinder -schemafinder foreign_key_bugfix false -s schemafinder -noshell -noinput -eval '
  true = register (hello, self ()),
  { atomic, ok } = delirium:heartbeat (),
  { atomic, ok } = mnesia:add_table_copy (flass, node (), ram_copies),
  true = lists:keymember (schemafinder, 1, application:which_applications ()),
  receive { From, ruthere } -> From ! imok end,
  receive after infinity -> ok end
' -s erlang halt &

wait_for_node turgy$pid

# now restart flassy ... *without* removing it's schema directory

echo "****** the following error is expected ******"

erl -pa ../src -setcookie $cook -sname flassy$pid -s combonodefinder -schemafinder foreign_key_bugfix false -s schemafinder -noshell -noinput -eval '
  false = lists:keymember (schemafinder, 1, application:which_applications ()),
  Node = node (),
  { aborted, { node_not_running, Node } } = 
    mnesia:add_table_copy (flass, node (), ram_copies)
' -s erlang halt || exit 1

echo "****** subsequent errors are not expected ******"

# oops, i'm an ardtard

rm -rf Mnesia*flassy*

erl -pa ../src -setcookie $cook -sname flassy$pid -s combonodefinder -schemafinder foreign_key_bugfix false -s schemafinder -noshell -noinput -eval '
  true = register (hello, self ()),
  { atomic, ok } = delirium:heartbeat (),
  { atomic, ok } = mnesia:add_table_copy (flass, node (), ram_copies),
  true = lists:keymember (schemafinder, 1, application:which_applications ()),
  receive { From, ruthere } -> From ! imok end,
  receive after infinity -> ok end
' -s erlang halt &

wait_for_node flassy$pid

} 2>&1 > test-disaster.out

exit 0
